from decimal import Decimal

import numpy as np
import pandas as pd

pd.set_option("display.max_columns", 100)
pd.set_option("display.max_rows", 1000)


# 清洗
def clearData():
    big_data = pd.read_csv("./original_data/roll_大数据分析.csv", encoding="GBK")
    computer_vision = pd.read_csv("./original_data/roll_计算机视觉.csv", encoding="GBK")
    database_management = pd.read_csv("./original_data/roll_数据库管理.csv", encoding="GBK")
    data_mining = pd.read_csv("./original_data/roll_大数据分析.csv", encoding="GBK")
    image_algorithms = pd.read_csv("./original_data/roll_图像算法.csv", encoding="GBK")
    data = pd.concat([big_data, computer_vision, database_management, data_mining, image_algorithms]).reset_index(
        drop=True)

    data["链接"] = data["el_链接"]
    data["地区"] = data["d"]
    data["薪资"] = data["sal"]
    del data["el_链接"]
    del data["d"]
    del data["sal"]

    data["地区"] = data["地区"].apply(area)
    data["时间"] = data["时间"].apply(date)
    data["薪资"] = data["薪资"].apply(salary)
    data = data.dropna(axis=0, how='any', thresh=None, subset=None, inplace=False)

    data.to_csv("./data/roll_data.csv", header=True, index=False)


# 地点
def area(data):
    area0 = data.split("|")
    area0 = area0[0]
    area0 = area0.split("-")
    area0 = area0[0]
    area0 = area0.strip()
    return area0


# 日期
def date(data):
    date0 = data.replace("发布", "")
    return date0

# 薪资
def salary(data):
    str1 = ""
    data = str(data)
    data = data.split("·")[0]
    if data[-1:] == "万":
        x = data.replace("万", "")
        x = x.split("-")
        for i in range(len(x)):
            x[i] = float(x[i]) * 10
            x[i] = Decimal(x[i]).quantize(Decimal("0.0"))
            x[i] = int(x[i])
            str1 += str(x[i]) + "-"
        return str1[:-1] + "K"
    elif data[-1:] == "千":
        x = data.replace("千", "")
        x = x.split("-")
        for i in range(len(x)):
            x[i] = float(x[i])
            x[i] = Decimal(x[i]).quantize(Decimal("0.0"))
            x[i] = int(x[i])
            str1 += str(x[i]) + "-"
        return str1[:-1] + "K"
    else:
        return np.nan


if __name__ == '__main__':
    clearData()
